import re 
import urllib2
import threading
import thread

class pageOps(threading.Thread):  
	def get_domain(self):
		if re.match("[a-zA-Z_]+://(.*)", self.url):
			return self.url[:self.url.find('/', self.url.find('://')+3)+1]
		return self.url[:self.url.find('/')+1]

	def get_source(self):
		req = urllib2.Request(url=self.url)
		res = urllib2.urlopen(req, timeout=10)
		return res.read()

	def correct_link(self, link):
		if re.match("#(.*)", link):
			return None
		if link.find('#') > 0:
			link = link[:link.find('#')]
		
		if re.match("[a-zA-Z_]+://(.*)", link):
			if link.find('/', 7) == -1:
				link = link + '/'
		elif re.match("//(.*)", link):
			link = "http:" + link
		elif re.match("/(.*)", link):
			link = self.domain[:-1] + link
		else:
			link = self.domain + link

		if not re.match("(.*?)://(.*?)\.(.*?)\.(.*)/(.*)", link):
			link = link[:link.find('://')] + "://www." + link[link.find('://')+3:]
		return link

	def get_links(self):
		links = []
		linksq = re.compile("<a(.*?)href='(.*?)'")
		linkdq = re.compile('<a(.*?)href="(.*?)"')
		for match in linksq.findall(self.source):
			links.append(self.correct_link(match[1]))
		for match in linkdq.findall(self.source):
			links.append(self.correct_link(match[1]))
		return links

	def get_content(self):
		content = {}
		
		titleTag = re.compile("<title>(.*?)</title>", re.I)
		bodyTag = re.compile("<body>(.*?)</body>", re.I)
		anyTag = re.compile("<(.*?)>(.*?)</(.*?)>")
		if titleTag.search(self.source):
			content["title"] = titleTag.findall(self.source)[0]
		else:
			content["title"] = self.url

		if bodyTag.search(self.source):
			body = bodyTag.findall(self.source)[0]
			bodyCont = ""
			for match in anyTag.split(body):
				if match:
					bodyCont += match
			content["body"] = bodyCont
		else:
			content["body"] = self.url

		return content

	def run(self):
		try:
			self.done = False
			self.domain = self.get_domain()
			self.source = self.get_source()
			self.content = self.get_content()
			self.links = self.get_links()
			self.done = True
			self.callback(self)
		except Exception as e:
			self.error(e, self.url)

	def __init__(self, url, callback, error):
		threading.Thread.__init__(self)
		self.url = url
		self._stop = threading.Event()
		self.callback = callback
		self.error = error
		self.daemon = True
		self.setName("pageCrawler - " + url)
		self.start()
